package edu.hawaii.webspider;

import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.logging.ConsoleHandler;
import java.util.logging.Level;
import java.util.logging.LogRecord;
import java.util.logging.Logger;

import com.meterware.httpunit.GetMethodWebRequest;
import com.meterware.httpunit.WebConversation;
import com.meterware.httpunit.WebRequest;
import com.meterware.httpunit.WebResponse;
import com.meterware.httpunit.WebLink;

/**
 * A class that uses HTTPUnit to crawl through pages and get the links on each page.  There are
 * three modes that this class can handle:
 * <ul>
 * <li>totallinks: Sums the total number of links found as it crawls through a given number
 *     of pages.</li>
 * <li>mostpopular: Goes through the given number of pages, then determines which of those has
 *     the most links pointing back to it.</li>
 * <li>logging: Turns on logging which prints the page it is retrieving and the number of links
 *     found on that page.</li>
 * </ul>
 * 
 * @author Philip Johnson
 * @author Laura Matsuo
 */
public class WebSpiderExample {

  /** Contains the starting URL for the crawl. */
  private String startUrl;
  /** Starting number of pages to crawl through. */
  private int startingNumPages;
  /** Number of pages to crawl through. */
  private int numPages;
  /** A hash containing all the visited pages and the number of links back to it. */
  private final Map<String, Integer> visitedLinks = new HashMap<String, Integer>();
  /** A hash containing all the accessible links and the number of times they appeared. */
  private final Map<String, Integer> allLinks = new HashMap<String, Integer>();
  /** The logger to display logging information. */
  private final Logger webSpiderLogger;
  /** The handler. */
  private final ConsoleHandler webSpiderHandler;
  /** Keeps track of the mode(s) to run in. */
  private boolean inTotalLinksMode = false;
  private boolean inMostPopularMode = false;
  private boolean inLoggingMode = false;
  /** The possible modes. */
  private static final String TOTAL_LINKS = "-totallinks";
  private static final String MOST_POPULAR = "-mostpopular";
  private static final String LOGGING = "-logging";
  /** The most popular URL. */
  private String mostPopularUrl;
  /** The number of pages pointing to the most popular URL. */
  private int mostPopularUrlCount;

  /**
   * Creates an empty WebSpiderExample object.
   */
  public WebSpiderExample() {
    // set the starting URL
    this.startUrl = "";
    // set the number of pages to 1
    this.startingNumPages = this.numPages = 1;
    // set up the logger
    this.webSpiderLogger = Logger.getLogger("edu.hawaii.webspider");
    this.webSpiderLogger.setUseParentHandlers(false);
    this.webSpiderHandler = new ConsoleHandler();
    this.webSpiderHandler.setFormatter(new WebSpiderFormatter());
    this.webSpiderLogger.addHandler(webSpiderHandler);
  }

  /**
   * Creates a WebSpiderExample object with a starting URL.
   * 
   * @param startUrl  The starting URL for the crawl.
   */
  public WebSpiderExample(String startUrl) {
    // set the starting URL
    this.startUrl = startUrl;
    // set the number of pages to 1
    this.startingNumPages = this.numPages = 1;
    // set up the logger
    this.webSpiderLogger = Logger.getLogger("edu.hawaii.webspider");
    this.webSpiderLogger.setUseParentHandlers(false);
    this.webSpiderHandler = new ConsoleHandler();
    this.webSpiderHandler.setFormatter(new WebSpiderFormatter());
    this.webSpiderLogger.addHandler(webSpiderHandler);
  }

  /**
   * Creates a WebSpiderExample object with a start URL and the number of pages to crawl through.
   * 
   * @param startUrl  The starting URL for the crawl.
   * @param numPages  The number of pages to crawl through.
   */
  public WebSpiderExample(String startUrl, int numPages) {
    // set the starting URL
    this.startUrl = startUrl;
    // set the number of pages
    this.startingNumPages = this.numPages = numPages;
    // set up the logger
    this.webSpiderLogger = Logger.getLogger("edu.hawaii.webspider");
    this.webSpiderLogger.setUseParentHandlers(false);
    this.webSpiderHandler = new ConsoleHandler();
    this.webSpiderHandler.setFormatter(new WebSpiderFormatter());
    this.webSpiderLogger.addHandler(webSpiderHandler);
  }
  
  /**
   * Gets the most popular URL.
   * 
   * @return  The visited URL with the most links pointing to it.
   */
  public String getMostPopularUrl() {
    return this.mostPopularUrl;
  }
  
  /**
   * Gets the number of links pointing to the most popular URL.
   * 
   * @return  The number of links pointing to the most popular URL.
   */
  public int getMostPopularUrlCount() {
    return this.mostPopularUrlCount;
  }
  
  /**
   * Sets the starting URL.
   * 
   * @param startUrl  The URL to start the web spider at.
   */
  public void setStartUrl(String startUrl) {
    this.startUrl = startUrl;
  }
  
  /**
   * Gets the starting URL.
   * 
   * @return  The starting URL.
   */
  public String getStartUrl() {
    return this.startUrl;
  }
  
  /**
   * Sets the remaining number of pages to crawl through.
   * 
   * @param numPages  The remaining number of pages the web spider should crawl through.
   */
  public void setNumPages(int numPages) {
    this.numPages = numPages;
  }
  
  /**
   * Gets the remaining number of pages to crawl through.
   * 
   * @return  The remaining number of pages to crawl through.
   */
  public int getNumPages() {
    return this.numPages;
  }
  
  /**
   * Sets the starting number of pages to crawl through.
   * 
   * @param startingNumPages  The starting number of pages to crawl through.
   */
  public void setStartingNumPages(int startingNumPages) {
    this.startingNumPages = startingNumPages;
  }
  
  /**
   * Gets the starting number of pages to crawl through.
   * 
   * @return  The starting number of pages to crawl through.
   */
  public int getStartingNumPages() {
    return this.startingNumPages;
  }
  
  /**
   * Determines whether the program is in total links mode.
   * 
   * @return  True if it is in total links mode, otherwise false. 
   */
  public boolean getInTotalLinksMode() {
    return inTotalLinksMode;
  }
  
  /**
   * Determines whether the program is in most popular mode.
   * 
   * @return  True if it is in most popular mode, otherwise false.
   */
  public boolean getInMostPopularMode() {
    return inMostPopularMode;
  }
  
  /**
   * Determines whether the program is in logging mode.
   * 
   * @return  True if it is in logging mode, otherwise false.
   */
  public boolean getInLoggingMode() {
    return inLoggingMode;
  }

  /**
   * Completes the Web Spider's set up by setting the start URL and the number of pages to crawl
   * through.
   * 
   * @param startUrl  The home URL for the crawl.
   * @param numPages  The number of pages to crawl through.
   */
  public void setUpWebSpider(String startUrl, int numPages) {
    this.startUrl = startUrl;
    this.startingNumPages = this.numPages = numPages;
  }

  /**
   * Returns the number of links found at the startUrl.
   * 
   * @return The number of links.
   * @throws Exception if problems occur retrieving the startUrl.
   */
  public int getNumLinks() throws Exception {
    // create the conversation object which will maintain state for us
    WebConversation wc = new WebConversation();
    WebRequest request = new GetMethodWebRequest(this.startUrl);
    WebResponse response = wc.getResponse(request);
    return response.getLinks().length;
  }

  /**
   * Gets the total number of pages accessible within the first given number of pages of a URL.
   * 
   * @param currentUrl  The URL to get the number of links from.
   * @return            The number of links available on the currentUrl page.
   */
  public int getTotalLinks(String currentUrl) {
    String urlString;
    int count = 0;
    WebLink[] links;

    this.printLogMessage("Retrieving " + currentUrl);
    this.visitedLinks.put(currentUrl, 1);

    if (numPages <= 0) {
      count = 0;
    }
    else {
      try {
        // get all links available from current URL
        WebConversation wc = new WebConversation();
        WebRequest request = new GetMethodWebRequest(currentUrl);
        WebResponse response = wc.getResponse(request);
        links = response.getLinks();

        this.printLogMessage("Found " + links.length + " links");

        // if there are any links on the current page
        if (links.length > 0) {
          // examine each link
          for (WebLink link : links) {
            // get the String value of the link
            urlString = link.getRequest().getURL().toString();
            
            // if the link is already in the visitedLinks hash
            if (this.visitedLinks.containsKey(urlString)) {
              this.visitedLinks.put(urlString, this.visitedLinks.get(urlString) + 1);
            }

            // if the link is already in the allLinks hash
            if (this.allLinks.containsKey(urlString)) {
              this.allLinks.put(urlString, this.allLinks.get(urlString) + 1);
            }
            // else if the link is not in the hash
            else {
              this.allLinks.put(urlString, 1);
              if (this.numPages > 0) {
                this.numPages--;
              }
              count = this.getTotalLinks(urlString);
            }
          }
        }
        // else there were no links on the current page
        else {
          count = 0;
        }
      }
      catch (Exception e) {
        System.out.println("Exception: " + e);
      }
    }

    return count;
  }

  /**
   * Calculates the total number of links accessible based on the hash containing the links and
   * counts. 
   * 
   * @return  The total number of links found.
   */
  public int sumNumberLinks() {
    Collection<Integer> linkCounts = this.allLinks.values();
    int sum = 0;
    for (Integer count : linkCounts) {
      sum += count;
    }
    return sum;
  }

  /**
   * Finds the first URL of those that were visited with the highest count.
   */
  public void findMostPopular() {
    this.mostPopularUrlCount = Integer.MIN_VALUE;
    Set<String> allUrls = this.visitedLinks.keySet();

    for (String url : allUrls) {
      if (this.visitedLinks.get(url) > this.mostPopularUrlCount) {
        this.mostPopularUrl = url;
        this.mostPopularUrlCount = this.visitedLinks.get(url);
      }
    }
  }

  /**
   * Sends the log message to be displayed if the program is in logging mode.
   * 
   * @param message  The message to be displayed.
   */
  public void printLogMessage(String message) {
    if (inLoggingMode) {
      webSpiderLogger.log(new LogRecord(Level.INFO, message));
    }
  }
  
  /**
   * Prints the usage info.
   */
  public void printUsage() {
    System.out.println("\n\nSorry, something was not initialized correctly");
    System.out.println("Usage is as follows: ");
    System.out.println("java -jar webspider-lauramat.jar -totallinks [complete starting URL] "
        + "[number of pages to crawl through]");
    System.out.println("java -jar webspider-lauramat.jar -mostpopular [complete starting URL] "
        + "[number of pages to crawl through]");
    System.out.println("java -jar webspider-lauramat.jar -totallinks [complete starting URL] "
        + "[number of pages to crawl through] -logging");
    System.out.println("java -jar webspider-lauramat.jar -mostpopular [complete starting URL] "
        + "[number of pages to crawl through] -logging");
  }
  
  /**
   * Goes through the args array to determine what mode(s) to run in and get the starting URL and
   * number of pages to go through.
   *  
   * @param args  Array containing the mode(s) to run in, starting URL, and number of pages.
   */
  public void setModes(String[] args) {
    for (String arg : args) {
      // determines the mode(s) to run in  
      if (TOTAL_LINKS.equalsIgnoreCase(arg)) {
        this.inTotalLinksMode = true;
      }
      else if (MOST_POPULAR.equalsIgnoreCase(arg)) {
        this.inMostPopularMode = true;
      }
      else if (LOGGING.equalsIgnoreCase(arg)) {
        this.inLoggingMode = true;
      }
      // sets the number of pages and start URL 
      else {
        try {
          this.setNumPages(Integer.valueOf(arg));
          this.setStartingNumPages(Integer.valueOf(arg));
        }
        catch (NumberFormatException e) {
          this.setStartUrl(arg);
        }
      }
    }
  }
  
  /**
   * Runs the web spider based on what modes were set.
   */
  public void run() {
    // run the program based on the mode
    if (this.inTotalLinksMode) {
      this.getTotalLinks(this.startUrl);
      System.out.println("\n\nThe total number of links discovered while crawling the first " 
          + this.startingNumPages + " pages accessible from " + startUrl + " is: " 
          + this.sumNumberLinks());
    }
    else if (this.inMostPopularMode) {
      this.getTotalLinks(this.startUrl);
      this.findMostPopular();
      System.out.println("\n\nThe most popular page discovered is: " + this.mostPopularUrl + 
          " with " + this.mostPopularUrlCount + " pages that point to it");
    }
    // if it doesn't know which mode it's in, print usage info
    else {
      this.printUsage();
    }
  }

  /**
   * Determines the mode(s) to run in, the starting URL, and the number of pages to crawl through
   * as it counts the number of links on each page it retrieves.
   * The program can run in the following modes:
   * <ul>
   * <li>totallinks: Sums the total number of links found as it crawls through a given number
   *     of pages.</li>
   * <li>mostpopular: Goes through the given number of pages, then determines which of those has
   *     the most links pointing back to it.</li>
   * <li>logging: Turns on logging which prints the page it is retrieving and the number of links
   *     found on that page.</li>
   * </ul>
   * 
   * @param args        Contains the mode(s) to run in, the starting URL, and the number of pages
   *                    to crawl through. 
   * @throws Exception  If problems occur.
   */
  public static void main(String[] args) throws Exception {
    WebSpiderExample webSpider = new WebSpiderExample();
    webSpider.setModes(args);
    webSpider.run();
  }
}
